{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "import evaluate"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate使用指南"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 查看支持的评估函数"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# Evaluate 库中的各种评估指标（metrics）,用于发现可用的评估指标名称\n",
    "all_modules =evaluate.list_evaluation_modules()\n",
    "all_modules"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "#列出所有官方的 comparison 类型评估模块的详细信息\n",
    "evaluate.list_evaluation_modules(\n",
    "\tmodule_type = \"comparison\", #只显示比较类型的评估指标\n",
    "\tinclude_community = False,  # 是否包含社区贡献的模块\n",
    "\twith_details = True)        # 显示详细信息"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载评估函数"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "accuracy = evaluate.load(\"accuracy\", download_mode = \"force_redownload\")",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "accuracy"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 查看函数说明"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "print(accuracy.description)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "print(accuracy.inputs_description)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "accuracy"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估指标计算——全局计算"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "accuracy = evaluate.load(\"accuracy\")\n",
    "results = accuracy.compute(references = [0, 1, 2, 0, 1, 2], predictions = [0, 1, 1, 2, 1, 0])\n",
    "results"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估指标计算——迭代计算"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "accuracy = evaluate.load(\"accuracy\")\n",
    "for ref, pred in zip([0, 1, 0, 1], [1, 0, 0, 1]):\n",
    "\taccuracy.add(references = ref, predictions = pred)\n",
    "accuracy.compute()"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "accuracy = evaluate.load(\"accuracy\")\n",
    "for refs, preds in zip([[0, 1], [0, 1]], [[1, 0], [0, 1]]):\n",
    "\taccuracy.add_batch(references = refs, predictions = preds)\n",
    "accuracy.compute()"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多个评估指标计算"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "clf_metrics = evaluate.combine([\"accuracy\", \"f1\", \"recall\", \"precision\"])\n",
    "clf_metrics"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "clf_metrics"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "clf_metrics.compute(predictions = [0, 1, 0], references = [0, 1, 1])",
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估结果对比可视化"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "from evaluate.visualization import radar_plot  # 目前只支持雷达图",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "data = [\n",
    "\t{\"accuracy\": 0.99, \"precision\": 0.8, \"f1\": 0.95, \"latency_in_seconds\": 33.6},\n",
    "\t{\"accuracy\": 0.98, \"precision\": 0.87, \"f1\": 0.91, \"latency_in_seconds\": 11.2},\n",
    "\t{\"accuracy\": 0.98, \"precision\": 0.78, \"f1\": 0.88, \"latency_in_seconds\": 87.6},\n",
    "\t{\"accuracy\": 0.88, \"precision\": 0.78, \"f1\": 0.81, \"latency_in_seconds\": 101.6}\n",
    "]\n",
    "model_names = [\"Model 1\", \"Model 2\", \"Model 3\", \"Model 4\"]"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "plot = radar_plot(data = data, model_names = model_names)",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
